{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PyTorch: Tensors\n",
    "----------------\n",
    "\n",
    "A fully-connected ReLU network with one hidden layer and no biases, trained to\n",
    "predict y from x by minimizing squared Euclidean distance.\n",
    "\n",
    "This implementation uses PyTorch tensors to manually compute the forward pass,\n",
    "loss, and backward pass.\n",
    "\n",
    "A PyTorch Tensor is basically the same as a numpy array: it does not know\n",
    "anything about deep learning or computational graphs or gradients, and is just\n",
    "a generic n-dimensional array to be used for arbitrary numeric computation.\n",
    "\n",
    "The biggest difference between a numpy array and a PyTorch Tensor is that\n",
    "a PyTorch Tensor can run on either CPU or GPU. To run operations on the GPU,\n",
    "just cast the Tensor to a cuda datatype.\n",
    "\n",
    "\n",
    "Source Link: http://pytorch.org/tutorials/beginner/examples_tensor/two_layer_net_tensor.html\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"background-image: linear-gradient( 135deg, #ABDCFF 10%, #0396FF 100%);\"> Orinal Tutorial code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 41622483.414673686\n",
      "1 41630199.54511545\n",
      "2 40851071.91791823\n",
      "3 33105787.059349153\n",
      "4 20981086.299429696\n",
      "5 10815450.644586504\n",
      "6 5324104.196151307\n",
      "7 2903010.1261253506\n",
      "8 1865730.5767264464\n",
      "9 1365004.0784591644\n",
      "10 1077363.4947750606\n",
      "11 884605.4597355041\n",
      "12 741801.907782577\n",
      "13 629894.7365108808\n",
      "14 539560.8456024791\n",
      "15 465369.4478917321\n",
      "16 403748.4962237244\n",
      "17 352059.34832199826\n",
      "18 308384.0736462213\n",
      "19 271257.60394115094\n",
      "20 239509.04533082037\n",
      "21 212255.30510675645\n",
      "22 188771.5376182426\n",
      "23 168374.9492944603\n",
      "24 150619.5903551164\n",
      "25 135084.0484521102\n",
      "26 121443.72228207812\n",
      "27 109431.39686890424\n",
      "28 98814.93577664904\n",
      "29 89411.36663325276\n",
      "30 81078.00257919323\n",
      "31 73655.73523676468\n",
      "32 67019.55546070288\n",
      "33 61081.673610930215\n",
      "34 55764.32052604947\n",
      "35 50985.061373530945\n",
      "36 46680.57501417854\n",
      "37 42797.91093505442\n",
      "38 39293.44032754915\n",
      "39 36118.797868516434\n",
      "40 33240.15068254639\n",
      "41 30626.509810968535\n",
      "42 28250.577162079047\n",
      "43 26084.540555213174\n",
      "44 24107.744586226978\n",
      "45 22302.829157841217\n",
      "46 20651.73625880845\n",
      "47 19139.98587668722\n",
      "48 17752.892838633685\n",
      "49 16479.786367603516\n",
      "50 15310.066838873667\n",
      "51 14234.634377899638\n",
      "52 13243.571748070251\n",
      "53 12329.998030616629\n",
      "54 11487.095859714973\n",
      "55 10709.487293395188\n",
      "56 9990.285145287804\n",
      "57 9324.588041292373\n",
      "58 8708.671432854986\n",
      "59 8137.700401703449\n",
      "60 7608.443722982971\n",
      "61 7116.875817706899\n",
      "62 6660.266905761371\n",
      "63 6236.208947609302\n",
      "64 5841.7053218133515\n",
      "65 5474.747177895057\n",
      "66 5132.8821135111375\n",
      "67 4814.487837528184\n",
      "68 4517.718365521314\n",
      "69 4240.947552226498\n",
      "70 3982.577227382106\n",
      "71 3741.2701803799246\n",
      "72 3515.775122638018\n",
      "73 3305.015515786128\n",
      "74 3107.9355920614753\n",
      "75 2923.5150856690598\n",
      "76 2750.8597455827585\n",
      "77 2589.17750692859\n",
      "78 2437.644553210462\n",
      "79 2295.7724028434386\n",
      "80 2162.8316605911896\n",
      "81 2038.1332826630655\n",
      "82 1921.1142224587402\n",
      "83 1811.2589504521989\n",
      "84 1708.1474137882742\n",
      "85 1611.263487593024\n",
      "86 1520.204450284104\n",
      "87 1434.655924027291\n",
      "88 1354.2203786123841\n",
      "89 1278.6226468248806\n",
      "90 1207.4955509764623\n",
      "91 1140.5364704211408\n",
      "92 1077.5065295054446\n",
      "93 1018.1790789739781\n",
      "94 962.343030955703\n",
      "95 909.7344544968964\n",
      "96 860.1458133586664\n",
      "97 813.4163213214324\n",
      "98 769.3571548042199\n",
      "99 727.8512876286841\n",
      "100 688.6888044531113\n",
      "101 651.7235873916494\n",
      "102 616.8527648209354\n",
      "103 583.9311595129567\n",
      "104 552.8654209261058\n",
      "105 523.536727383454\n",
      "106 495.829926335136\n",
      "107 469.6739307211151\n",
      "108 444.94705184412373\n",
      "109 421.57816977846414\n",
      "110 399.51717372829853\n",
      "111 378.6509052244562\n",
      "112 358.91588412569274\n",
      "113 340.2557097652716\n",
      "114 322.6135252692909\n",
      "115 305.9269419431726\n",
      "116 290.14444280052635\n",
      "117 275.201344993701\n",
      "118 261.06177630845326\n",
      "119 247.67669918126245\n",
      "120 235.00589557474405\n",
      "121 223.01669736269935\n",
      "122 211.6580720487392\n",
      "123 200.89912081037482\n",
      "124 190.71060520845424\n",
      "125 181.05856923119927\n",
      "126 171.9092329265652\n",
      "127 163.24389440423994\n",
      "128 155.02997713592637\n",
      "129 147.24006045791862\n",
      "130 139.85530971354063\n",
      "131 132.85718592515968\n",
      "132 126.22025241005315\n",
      "133 119.93347370094534\n",
      "134 113.96603981644297\n",
      "135 108.30188434273404\n",
      "136 102.92890543419384\n",
      "137 97.83291986063375\n",
      "138 92.9974520812475\n",
      "139 88.40883135152694\n",
      "140 84.05158067815253\n",
      "141 79.91849090744623\n",
      "142 75.99297856744997\n",
      "143 72.26645647313745\n",
      "144 68.72676169921576\n",
      "145 65.3683520574987\n",
      "146 62.17801694278178\n",
      "147 59.14610364492465\n",
      "148 56.26744822170367\n",
      "149 53.532291242796205\n",
      "150 50.93355532438643\n",
      "151 48.46705068044173\n",
      "152 46.12081309159704\n",
      "153 43.89176981550651\n",
      "154 41.772817744795944\n",
      "155 39.75776101329416\n",
      "156 37.84461183534961\n",
      "157 36.02451560630625\n",
      "158 34.29478185208761\n",
      "159 32.65082412936417\n",
      "160 31.086314408274347\n",
      "161 29.598940031029997\n",
      "162 28.18359662897806\n",
      "163 26.838315341402105\n",
      "164 25.559100431529444\n",
      "165 24.341798489895353\n",
      "166 23.183071356748485\n",
      "167 22.08172797578422\n",
      "168 21.033989098136175\n",
      "169 20.03675132284434\n",
      "170 19.087508395625235\n",
      "171 18.18425457980068\n",
      "172 17.325156882854944\n",
      "173 16.506762195826585\n",
      "174 15.72826897664402\n",
      "175 14.986654310774057\n",
      "176 14.281201595165246\n",
      "177 13.609941013194632\n",
      "178 12.9704420467055\n",
      "179 12.361173318729584\n",
      "180 11.78117972697001\n",
      "181 11.2293027221655\n",
      "182 10.703219552660308\n",
      "183 10.202733420376418\n",
      "184 9.725890240859773\n",
      "185 9.271756566077286\n",
      "186 8.83953915537596\n",
      "187 8.427542551304771\n",
      "188 8.034864011453326\n",
      "189 7.660934486671255\n",
      "190 7.304632211651102\n",
      "191 6.964960408105783\n",
      "192 6.641651673064016\n",
      "193 6.333560292376363\n",
      "194 6.039580022311075\n",
      "195 5.75993463642088\n",
      "196 5.493466925649248\n",
      "197 5.239273681168104\n",
      "198 4.996964677324801\n",
      "199 4.7662361811296705\n",
      "200 4.546332761439487\n",
      "201 4.336397831569446\n",
      "202 4.136523328408586\n",
      "203 3.946070346953178\n",
      "204 3.7642122688922974\n",
      "205 3.5912686784316747\n",
      "206 3.426008226321713\n",
      "207 3.2685250648729927\n",
      "208 3.118437855207887\n",
      "209 2.975378962089696\n",
      "210 2.83892080576301\n",
      "211 2.708873296407816\n",
      "212 2.5845871749603617\n",
      "213 2.4664273006406603\n",
      "214 2.3536949620028196\n",
      "215 2.2458765963578458\n",
      "216 2.1432977584875346\n",
      "217 2.0453768327174373\n",
      "218 1.952028212300334\n",
      "219 1.8630844471804409\n",
      "220 1.7779136950551795\n",
      "221 1.696978990995845\n",
      "222 1.6195978993386042\n",
      "223 1.545902954085367\n",
      "224 1.4756192121212077\n",
      "225 1.4083921819267395\n",
      "226 1.3442975871554097\n",
      "227 1.2831306418075705\n",
      "228 1.2249843472010227\n",
      "229 1.169232850204832\n",
      "230 1.1162130034230486\n",
      "231 1.0656274946514916\n",
      "232 1.017319385484993\n",
      "233 0.9711467263720941\n",
      "234 0.9271633799098176\n",
      "235 0.8852168877875535\n",
      "236 0.8451731169454675\n",
      "237 0.8069043315620306\n",
      "238 0.7705113687499292\n",
      "239 0.7355726767966111\n",
      "240 0.7023756402017582\n",
      "241 0.6705827512133169\n",
      "242 0.6403754725620985\n",
      "243 0.6115097639780815\n",
      "244 0.5838317665732378\n",
      "245 0.5575264229254202\n",
      "246 0.5323817269978477\n",
      "247 0.508428923471894\n",
      "248 0.4855187205935261\n",
      "249 0.4636607279039291\n",
      "250 0.4427929253837135\n",
      "251 0.4229131990910153\n",
      "252 0.4038682377928533\n",
      "253 0.38567520548161127\n",
      "254 0.3683486428569136\n",
      "255 0.35182470344993977\n",
      "256 0.3360063112672371\n",
      "257 0.320890904980506\n",
      "258 0.3064776427097229\n",
      "259 0.2927596380090236\n",
      "260 0.27960051330390456\n",
      "261 0.26707547511827556\n",
      "262 0.2550709286815902\n",
      "263 0.24359104600205406\n",
      "264 0.23271503279620465\n",
      "265 0.22225992311441822\n",
      "266 0.21229866324035118\n",
      "267 0.2027998345204054\n",
      "268 0.1937602694898226\n",
      "269 0.18509506737119352\n",
      "270 0.17679788569253163\n",
      "271 0.16888735963957302\n",
      "272 0.16133530066571944\n",
      "273 0.15414855811662154\n",
      "274 0.14724559823176975\n",
      "275 0.140675919437923\n",
      "276 0.13440097838170306\n",
      "277 0.1283883629774074\n",
      "278 0.12265151802236574\n",
      "279 0.11720907403880965\n",
      "280 0.11198544584067704\n",
      "281 0.10697507212594104\n",
      "282 0.1022194447195055\n",
      "283 0.09767717854088165\n",
      "284 0.09336047219445387\n",
      "285 0.08917259607098416\n",
      "286 0.0851898182524522\n",
      "287 0.08141788260161098\n",
      "288 0.07780817490780387\n",
      "289 0.07433562733547916\n",
      "290 0.071021995107532\n",
      "291 0.06786235892874035\n",
      "292 0.06486875972360195\n",
      "293 0.06194793139136623\n",
      "294 0.05918055497156027\n",
      "295 0.05657181964854019\n",
      "296 0.05408213397940642\n",
      "297 0.0516823052352936\n",
      "298 0.049378941151710265\n",
      "299 0.04717551068140036\n",
      "300 0.0451018268654374\n",
      "301 0.04307964942040643\n",
      "302 0.04119053979335896\n",
      "303 0.03937893098069467\n",
      "304 0.03764445369437763\n",
      "305 0.035968444855715376\n",
      "306 0.03438586443743441\n",
      "307 0.0328514722882014\n",
      "308 0.031411824231678764\n",
      "309 0.030021517907402817\n",
      "310 0.028701351436278166\n",
      "311 0.027430168176326042\n",
      "312 0.02622875663004809\n",
      "313 0.025077597128162488\n",
      "314 0.02397150249805957\n",
      "315 0.02290144280361872\n",
      "316 0.021895646693179338\n",
      "317 0.02093648036577811\n",
      "318 0.020021968111014754\n",
      "319 0.019141381507384203\n",
      "320 0.018306091541213654\n",
      "321 0.017508204430428886\n",
      "322 0.016744370018350496\n",
      "323 0.016014340082312506\n",
      "324 0.01531058548781683\n",
      "325 0.014643868525223258\n",
      "326 0.014005781112498505\n",
      "327 0.013400394473146893\n",
      "328 0.012815979457833748\n",
      "329 0.01226305582928805\n",
      "330 0.011736811281905846\n",
      "331 0.011225136210822662\n",
      "332 0.010737596766252244\n",
      "333 0.01027760819575041\n",
      "334 0.009831547856204104\n",
      "335 0.009416002823384284\n",
      "336 0.009008381409738142\n",
      "337 0.00862294185625978\n",
      "338 0.008250966720144692\n",
      "339 0.00789770651940569\n",
      "340 0.007563675040759055\n",
      "341 0.007242924534535122\n",
      "342 0.006932506759710355\n",
      "343 0.00664525299420049\n",
      "344 0.006367970982865678\n",
      "345 0.006096062394444157\n",
      "346 0.0058353056601524456\n",
      "347 0.005598113646044345\n",
      "348 0.005355671002101436\n",
      "349 0.005133848366208671\n",
      "350 0.004924045657939802\n",
      "351 0.004719941412014705\n",
      "352 0.004528182322880658\n",
      "353 0.004341523763309207\n",
      "354 0.004168514231200682\n",
      "355 0.003996473151937883\n",
      "356 0.0038342885276765326\n",
      "357 0.0036755472749678483\n",
      "358 0.003531627121877684\n",
      "359 0.003389818886140627\n",
      "360 0.0032559895426353958\n",
      "361 0.0031267679865517595\n",
      "362 0.003002050701386816\n",
      "363 0.002883330125092301\n",
      "364 0.0027681180002565586\n",
      "365 0.0026597054141610788\n",
      "366 0.002556765058788716\n",
      "367 0.002457673049765685\n",
      "368 0.002364488963974065\n",
      "369 0.002273691790715626\n",
      "370 0.0021854780900890936\n",
      "371 0.0021037085282330503\n",
      "372 0.002025151149296252\n",
      "373 0.001947775353783307\n",
      "374 0.0018734438047230961\n",
      "375 0.0018056585972634764\n",
      "376 0.0017394397018805352\n",
      "377 0.0016715388973929188\n",
      "378 0.0016121349487736025\n",
      "379 0.0015531765044841696\n",
      "380 0.0014985420232962038\n",
      "381 0.0014472255444680539\n",
      "382 0.0013946744824721719\n",
      "383 0.0013485035931028455\n",
      "384 0.0013015798672537937\n",
      "385 0.0012564521092326003\n",
      "386 0.0012132691745448537\n",
      "387 0.0011714132951741868\n",
      "388 0.0011313743464567462\n",
      "389 0.0010936307177019378\n",
      "390 0.0010563395968588157\n",
      "391 0.001019917231244949\n",
      "392 0.0009871605506042713\n",
      "393 0.0009550284825222133\n",
      "394 0.0009254644915709931\n",
      "395 0.0008946513434865723\n",
      "396 0.0008643721654887256\n",
      "397 0.0008380448514984651\n",
      "398 0.0008116476878115564\n",
      "399 0.0007846151911017896\n",
      "400 0.0007624668080010305\n",
      "401 0.0007388613696005519\n",
      "402 0.0007161120825172063\n",
      "403 0.0006940291959851019\n",
      "404 0.0006737901233501153\n",
      "405 0.0006542206037053466\n",
      "406 0.0006338645153120132\n",
      "407 0.0006158757266874898\n",
      "408 0.0005982314171230213\n",
      "409 0.0005818609514282591\n",
      "410 0.0005646341750906725\n",
      "411 0.0005477579763775164\n",
      "412 0.0005314752012805513\n",
      "413 0.000517954278997132\n",
      "414 0.0005028296672191512\n",
      "415 0.0004887103076793853\n",
      "416 0.0004771757114262476\n",
      "417 0.00046330820985808485\n",
      "418 0.00045050874130661556\n",
      "419 0.00043874155595280406\n",
      "420 0.0004269810667149182\n",
      "421 0.0004147014641229796\n",
      "422 0.00040412213105466\n",
      "423 0.0003947733857968816\n",
      "424 0.0003844010529616554\n",
      "425 0.0003745382142144732\n",
      "426 0.0003644591657918639\n",
      "427 0.00035602112395332133\n",
      "428 0.00034616150134815504\n",
      "429 0.0003380740988072617\n",
      "430 0.0003296757735634266\n",
      "431 0.000321901666345617\n",
      "432 0.00031388293203887807\n",
      "433 0.0003052872629608072\n",
      "434 0.0002982089033351093\n",
      "435 0.00029205397091214627\n",
      "436 0.00028507254577484625\n",
      "437 0.000278127533295619\n",
      "438 0.00027152710470415564\n",
      "439 0.00026504980813688364\n",
      "440 0.00025901695796282445\n",
      "441 0.00025269634250882356\n",
      "442 0.00024682758640430513\n",
      "443 0.0002414723806120206\n",
      "444 0.00023620694806993428\n",
      "445 0.0002308898850409552\n",
      "446 0.00022604853290784932\n",
      "447 0.00022137566929383845\n",
      "448 0.00021649355076961557\n",
      "449 0.00021206349892724866\n",
      "450 0.0002063092373336961\n",
      "451 0.00020299958358523107\n",
      "452 0.00019871822247598606\n",
      "453 0.0001941004896871268\n",
      "454 0.00019007339648246038\n",
      "455 0.00018647573793888553\n",
      "456 0.00018204828023501984\n",
      "457 0.00017857486361069763\n",
      "458 0.00017489158002628735\n",
      "459 0.00017137775343120665\n",
      "460 0.0001685236191249391\n",
      "461 0.00016527896738152692\n",
      "462 0.00016176576740416573\n",
      "463 0.00015860747578231096\n",
      "464 0.0001556005547077889\n",
      "465 0.00015247972090975948\n",
      "466 0.00014957388426228846\n",
      "467 0.00014669578826441177\n",
      "468 0.0001438988007455011\n",
      "469 0.00014168173830603203\n",
      "470 0.00013920435223710795\n",
      "471 0.00013667100949178723\n",
      "472 0.00013384082268549924\n",
      "473 0.00013146168951336593\n",
      "474 0.00012881699034231263\n",
      "475 0.0001263524650170489\n",
      "476 0.0001235690983597479\n",
      "477 0.00012168042086149744\n",
      "478 0.00011951765092775568\n",
      "479 0.00011749181914344855\n",
      "480 0.0001154759789731008\n",
      "481 0.00011386267601628391\n",
      "482 0.00011186072477813516\n",
      "483 0.00011011143855474415\n",
      "484 0.0001082947327297884\n",
      "485 0.00010612684535014905\n",
      "486 0.00010434093711066475\n",
      "487 0.0001031435784834972\n",
      "488 0.0001013937870478071\n",
      "489 9.956797551995133e-05\n",
      "490 9.775865844686982e-05\n",
      "491 9.617084116449529e-05\n",
      "492 9.477100693002297e-05\n",
      "493 9.323471434244737e-05\n",
      "494 9.174551072624375e-05\n",
      "495 9.015678473752939e-05\n",
      "496 8.879942483677739e-05\n",
      "497 8.738818710370133e-05\n",
      "498 8.641726440286948e-05\n",
      "499 8.513837026024159e-05\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "dtype = torch.FloatTensor\n",
    "# dtype = torch.cuda.FloatTensor # Uncomment this to run on GPU\n",
    "\n",
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# Create random input and output data\n",
    "x = torch.randn(N, D_in).type(dtype)\n",
    "y = torch.randn(N, D_out).type(dtype)\n",
    "\n",
    "# Randomly initialize weights\n",
    "w1 = torch.randn(D_in, H).type(dtype)\n",
    "w2 = torch.randn(H, D_out).type(dtype)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # Forward pass: compute predicted y\n",
    "    h = x.mm(w1)\n",
    "    h_relu = h.clamp(min=0)\n",
    "    y_pred = h_relu.mm(w2)\n",
    "\n",
    "    # Compute and print loss\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss)\n",
    "\n",
    "    # Backprop to compute gradients of w1 and w2 with respect to loss\n",
    "    grad_y_pred = 2.0 * (y_pred - y)\n",
    "    grad_w2 = h_relu.t().mm(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.mm(w2.t())\n",
    "    grad_h = grad_h_relu.clone()\n",
    "    grad_h[h < 0] = 0\n",
    "    grad_w1 = x.t().mm(grad_h)\n",
    "\n",
    "    # Update weights using gradient descent\n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
